#if USE_NUMA == 1

#include <cassert>

#include <hip/hip_runtime.h>
#include <numa.h>

#include "scope/init/init.hpp"
#include "scope/utils/utils.hpp"
#include "scope/init/flags.hpp"

#include "args.hpp"
#include "init/flags.hpp"
#include "init/numa.hpp"
#include "utils/numa.hpp"

#define NAME "Comm/NUMAMemcpy/GPUToWC"

static void Comm_NUMAMemcpy_GPUToWC(benchmark::State &state) {

  if (!has_cuda) {
    state.SkipWithError(NAME " no CUDA device found");
    return;
  }

  if (!has_numa) {
    state.SkipWithError(NAME " NUMA not available");
    return;
  }

  const int numa_id = FLAG(numa_ids)[0];
  const int cuda_id = FLAG(cuda_device_ids)[0];

  const auto bytes  = 1ULL << static_cast<size_t>(state.range(0));


  numa_bind_node(numa_id);
  if (PRINT_IF_ERROR(utils::cuda_reset_device(cuda_id))) {
    state.SkipWithError(NAME " failed to reset CUDA device");
    return;
  }

  char *src = nullptr;
  char *dst = nullptr;
  if (PRINT_IF_ERROR(hipHostMalloc(&dst, bytes, hipHostMallocDefault))) {
    state.SkipWithError(NAME " failed to perform pinned hipHostMalloc");
    return;
  }
  defer(hipHostFree(dst));

  if (PRINT_IF_ERROR(hipSetDevice(cuda_id))) {
    state.SkipWithError(NAME " failed to set CUDA device");
    return;
  }

  if (PRINT_IF_ERROR(hipMalloc(&src, bytes))) {
    state.SkipWithError(NAME " failed to perform hipMalloc");
    return;
  }
  defer(hipFree(src));

  if (PRINT_IF_ERROR(hipMemset(src, 0, bytes))) {
    state.SkipWithError(NAME " failed to perform hipMemset");
    return;
  }

  hipEvent_t start, stop;
  PRINT_IF_ERROR(hipEventCreate(&start));
  PRINT_IF_ERROR(hipEventCreate(&stop));

  for (auto _ : state) {
    hipEventRecord(start, NULL);

    const auto cuda_err = hipMemcpy(dst, src, bytes, hipMemcpyDeviceToHost);

    hipEventRecord(stop, NULL);
    hipEventSynchronize(stop);

    if (PRINT_IF_ERROR(cuda_err) != hipSuccess) {
      state.SkipWithError(NAME " failed to perform memcpy");
      break;
    }

    float msecTotal = 0.0f;
    if (PRINT_IF_ERROR(hipEventElapsedTime(&msecTotal, start, stop))) {
      state.SkipWithError(NAME " failed to get elapsed time");
      break;
    }
    state.SetIterationTime(msecTotal / 1000);
  }
  state.SetBytesProcessed(int64_t(state.iterations()) * int64_t(bytes));
  state.counters.insert({{"bytes", bytes}});

  // reset to run on any node
  numa_bind_node(-1);
}

BENCHMARK(Comm_NUMAMemcpy_GPUToWC)->SMALL_ARGS()->UseManualTime();

#endif // USE_NUMA == 1
